<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
<%@ taglib prefix="pageContext" uri="http://java.sun.com/jsp/jstl/functions" %>
<%@ page import="servlet.news1"%>
<%@ page import="java.util.List" %>
<%@ page import="java.util.ArrayList" %>
<%@ page import="java.net.URL" %>
<%@ page import="java.net.URLConnection" %>
<%@ page import="java.net.HttpURLConnection" %>
<%@ page import="java.util.regex.Pattern" %>
<%@ page import="java.io.*" %>
<%@ page import="java.util.regex.Matcher" %>
<%@ page import="static java.lang.System.out" %>
<%@ page import="static java.lang.System.out" %>
<%@ page import="servlet.news" %>
<%--
  Created by IntelliJ IDEA.
  User: 小陈同学
  Date: 2021/1/2
  Time: 15:22
  To change this template use File | Settings | File Templates.
--%>
<%@ page contentType="text/html;charset=UTF-8" language="java" %>
<html>
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>新闻中心</title>
    <link rel="stylesheet" href="style.css">
</head>
<body>
<%
    List<String> urls1=new ArrayList<>();
    List<String> urls2=new ArrayList<>();
    //待爬取的URL
    String strurl="http://news.baidu.com/";
    //http://news.baidu.com/
    //http://www.xinhuanet.com/
    //建立url爬取核心对象
    try {
        URL url=new URL(strurl);
        //通过url建立与网页的连接
        URLConnection conn=url.openConnection();
        HttpURLConnection con=(HttpURLConnection)conn;

        con.setDoInput(true);
        con.setDoOutput(true);
        //伪装成浏览器访问，防止拦截
//            con.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
//            con.setRequestProperty("Accept-Language","zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
//            con.setRequestProperty("Cookie","BAIDUID=5CEA9CDFFD08E479448DFBC5B5E563EC:FG=1; BIDUPSID=5CEA9CDFFD08E479C5BD5A95C28469C1; PSTM=1585878066");
//            con.setRequestProperty("Host","news.baidu.com");
//            con.setRequestProperty("Referer","news.baidu.com");
//            con.setRequestProperty("Upgrade-Insecure-Requests","1");
//            //*****
//            con.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; AppleWebKit/537.36 (KHTML, like Gecko)) Chrome/87.0.4280.88 Safari/537.36");
//            con.setRequestProperty("Connection","keep-alive");
//            con.setUseCaches(false);
        int code=con.getResponseCode();
        if(200==code) {
            //通过链接取得网页返回的数据
            InputStream is=con.getInputStream();
            //按行读取网页数据，并进行内容分析
            //因此用BufferedReader和InputStreamReader把字节流转化为字符流的缓冲流
            //进行转换时，需要处理编码格式问题
            BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
            //按行读取并打印
            String line = null;
            boolean isStart=false;
            while ((line = br.readLine()) != null) {
                // System.out.println(line);
                if (line.contains("<div class=\"hotnews\" alog-group=\"focustop-hotnews\">"))
                    isStart = true;
                if(line.contains("div id=\"pane-recommend\" class=\"mod-tab-pane pane-recommend \""))
                    isStart=false;

//                    if (line.contains("<div class=\"part2L\" id=\"hpart2L\">"))
//                        isStart = true;
                //
                if (isStart) {
//                    System.out.println(line);
                    //正则表达式的匹配规则提取该网页的链接
                    Pattern p = Pattern.compile("<a .*href=.+</a>");
                    //建立一个输出流，用于保存文件,文件名为执行时间，以防重复
                    PrintWriter pw = new PrintWriter(new File("1.txt"));

                    while ((line = br.readLine()) != null) {
                        //System.out.println(line);
                        //编写正则，匹配超链接地址
                        pw.println(line);
                        Matcher m = p.matcher(line);
                        while (m.find()) {
                            String href = m.group();
                            //找到超链接地址并截取字符串
                            //有无引号
                            //匹配到标签后，分两步走
                            //1.获取标签内容
                            String regEx_html = "<[^>]+>"; //定义HTML标签的正则表达式
                            Pattern p_html = Pattern.compile(regEx_html,Pattern.CASE_INSENSITIVE);
                            Matcher m_html = p_html.matcher(href);
                            String htmlStr = m_html.replaceAll(""); //过滤html标签
                            //2.获取a 里面的href
                            out.println();
                            href = href.substring(href.indexOf("href="));
                            if (href.contains("http")) {
                                if (href.charAt(5) == '\"') {
                                    href = href.substring(6);
                                } else {
                                    href = href.substring(5);
                                }
                                //截取到引号或者空格或者到">"结束
                                try {
                                    href = href.substring(0, href.indexOf("\""));
                                } catch (Exception e) {
                                    try {
                                        href = href.substring(0, href.indexOf(" "));
                                    } catch (Exception e1) {
                                        href = href.substring(0, href.indexOf(">"));
                                    }
                                }
                               // out.println("新闻地址：" + href+",标题："+htmlStr);

                                if (href.contains("?id")){

                                    urls1.add(href);
                                    urls2.add(htmlStr);

                                }

                               // urls2.add(htmlStr);
                                pageContext.setAttribute("urls1",urls1);
                                pageContext.setAttribute("urls2",urls2);
                                news news=new news();
                                news.setTitle(htmlStr);
                                news.setUrl(href);
                                // NewsDao.insert(news);

                            }
                        }

                    }
                }
            }
            br.close();
//                int count=-1;
//                byte[] buff=new byte[1024];
//                FileOutputStream out=new FileOutputStream(new File("1.html"));
//                while((count=is.read(buff))!=-1)
//                {
//                    out.write(buff,0,count);
//                }
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
%>

<div class="header w">
    <div class="logo">
        <img src="images/logo.jpg" alt="">
    </div>
    <div class="nav">
        <ul>
            <li><a href="index.jsp" >首页</a></li>
            <li><a href="electapp.jsp">家电类</a></li>
            <li><a href="clothes.jsp" >衣物类</a></li>
            <li><a href="food.jsp" >食品类</a></li>
            <li><a href="medicine.jsp">医药类</a></li>
            <li><a href="news.jsp" class="a1">新闻中心</a></li>
            <li><a href="email.jsp">邮件反馈</a></li>
            <li><a href="reback.jsp">联系我们</a></li>
        </ul>
    </div>
</div>
    <div class="w">

        <div class="banner2" style="background:url(images/new1.jpg)">

        </div>

        <div class="banner1">
            <div class="n">

                <table border="1">

                    <tr>
                        <td>新闻标题</td>
                        <td>新闻地址</td>
                    </tr>
                    <tr>

                        <td>
                            <c:forEach items="${urls2}"  var="s1" step="1">
                                <br/>  ${s1 }<br/>
                            </c:forEach>
                        </td>
                        <td>
                            <c:forEach items="${urls1}" var="s2" step="1">
                                <br/>  ${s2 }<br/>
                            </c:forEach>
                        </td>
                    </tr>
<%--                    <%}%>--%>
                </table>




            </div>
        </div>






</div>
<!-- 尾部footer -->
<div class="footer ">
    <div class="w">
        <div class="copyright">
            <img src="images/logo.jpg" alt="">
        </div>
        <div class="links">
            <dl>
                <dt>帮助中心</dt>
                <dd><a href="login.jsp">用户登录</a></dd>
                <dd><a href="Register.jsp">用户注册</a></dd>
            </dl>
            <dl>
                <dt>网站介绍</dt>
                <dd><a>本网站以网上购物为中心，分别设置四大类商品进行出售。配有新闻界面可以了解时事政治。邮件联系可以发送邮件给我们以提出宝贵的建议</a></dd>

            </dl>
        </div>
    </div>
</div>
</body>

</html>